{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## todo: \n",
    "# 解决测评函数不能引入cv的问题 done\n",
    "# 建立cv流程进行调参 在模型处引入权重因素\n",
    "# 清理异常点(小于0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_test=pd.read_csv('input/train_time.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_test=train_test.fillna(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>college</th>\n",
       "      <th>rank</th>\n",
       "      <th>total_people</th>\n",
       "      <th>rank_percent</th>\n",
       "      <th>countM1</th>\n",
       "      <th>price_sumM1</th>\n",
       "      <th>price_avgM1</th>\n",
       "      <th>price_maxM1</th>\n",
       "      <th>...</th>\n",
       "      <th>地点263_avg</th>\n",
       "      <th>地点263_max</th>\n",
       "      <th>地点263_min</th>\n",
       "      <th>地点263_median</th>\n",
       "      <th>地点840_count</th>\n",
       "      <th>地点840_sum</th>\n",
       "      <th>地点840_avg</th>\n",
       "      <th>地点840_max</th>\n",
       "      <th>地点840_min</th>\n",
       "      <th>地点840_median</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2933.0</td>\n",
       "      <td>0.000341</td>\n",
       "      <td>49.0</td>\n",
       "      <td>201.31</td>\n",
       "      <td>4.108367</td>\n",
       "      <td>36.4</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2933.0</td>\n",
       "      <td>0.000682</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>6.157895</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1565.0</td>\n",
       "      <td>1570.0</td>\n",
       "      <td>0.996815</td>\n",
       "      <td>97.0</td>\n",
       "      <td>347.74</td>\n",
       "      <td>3.584948</td>\n",
       "      <td>10.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1570.0</td>\n",
       "      <td>1570.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>98.0</td>\n",
       "      <td>491.01</td>\n",
       "      <td>5.010306</td>\n",
       "      <td>17.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>0.000434</td>\n",
       "      <td>27.0</td>\n",
       "      <td>82.96</td>\n",
       "      <td>3.072593</td>\n",
       "      <td>22.3</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 432 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label  college    rank  total_people  rank_percent  countM1  \\\n",
       "0   0    0.0      9.0     1.0        2933.0      0.000341     49.0   \n",
       "1   1    0.0      9.0     2.0        2933.0      0.000682     -1.0   \n",
       "2   8    0.0      6.0  1565.0        1570.0      0.996815     97.0   \n",
       "3   9    0.0      6.0  1570.0        1570.0      1.000000     98.0   \n",
       "4  10    0.0      3.0     1.0        2304.0      0.000434     27.0   \n",
       "\n",
       "   price_sumM1  price_avgM1  price_maxM1      ...       地点263_avg  地点263_max  \\\n",
       "0       201.31     4.108367         36.4      ...            -1.0       -1.0   \n",
       "1        -1.00    -1.000000         -1.0      ...            -1.0       -1.0   \n",
       "2       347.74     3.584948         10.0      ...            -1.0       -1.0   \n",
       "3       491.01     5.010306         17.5      ...            -1.0       -1.0   \n",
       "4        82.96     3.072593         22.3      ...            -1.0       -1.0   \n",
       "\n",
       "   地点263_min  地点263_median  地点840_count  地点840_sum  地点840_avg  地点840_max  \\\n",
       "0       -1.0          -1.0         -1.0       -1.0  -1.000000       -1.0   \n",
       "1       -1.0          -1.0         19.0      117.0   6.157895        7.0   \n",
       "2       -1.0          -1.0         -1.0       -1.0  -1.000000       -1.0   \n",
       "3       -1.0          -1.0         -1.0       -1.0  -1.000000       -1.0   \n",
       "4       -1.0          -1.0          2.0        7.0   3.500000        4.0   \n",
       "\n",
       "   地点840_min  地点840_median  \n",
       "0       -1.0          -1.0  \n",
       "1        4.0           6.0  \n",
       "2       -1.0          -1.0  \n",
       "3       -1.0          -1.0  \n",
       "4        3.0           3.5  \n",
       "\n",
       "[5 rows x 432 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Evaluation \n",
    "def f1_macro(label_truth, predictions):\n",
    "    df=pd.DataFrame(columns=[\"subsidy_x\",\"subsidy_y\"])\n",
    "    df.subsidy_y=predictions\n",
    "    df.subsidy_x=np.array(label_truth)\n",
    "    df.subsidy_y = df.subsidy_y.apply(lambda x:int(x))\n",
    "\n",
    "    \n",
    "    correct = df[df['subsidy_x'] == df['subsidy_y']]\n",
    "    s = 0\n",
    "    for i in [1000, 1500, 2000]:\n",
    "        print '\\n%d'%i\n",
    "        if sum(df['subsidy_x'] == i)!=0:\n",
    "            r = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_x'] == i)\n",
    "            print 'Recall---%s'%r\n",
    "        else: \n",
    "            r=0\n",
    "        if sum(df['subsidy_y'] == i)!=0:\n",
    "            p = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_y'] == i)        \n",
    "            print 'Precision---%s'%p\n",
    "        else:\n",
    "            p=0\n",
    "        if (r+p)!=0:\n",
    "            f = r*p*2/(r+p)\n",
    "            print 'F1---%s'%f\n",
    "        if not np.isnan(f):\n",
    "            s += (float(sum(df['subsidy_x'] == i))/df.shape[0])*f\n",
    "    \n",
    "    print '\\nF1-macro---%s'%s\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = train_test.ix[:, train_test.columns != 'label']\n",
    "y = train_test.ix[:, train_test.columns == 'label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.3, random_state = 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  from ipykernel import kernelapp as app\n",
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "train=X_train\n",
    "train['label']=y_train\n",
    "\n",
    "test=X_test\n",
    "test['label']=y_test\n",
    "\n",
    "test.to_csv('input/test_7r07part.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nice_feature=pd.read_csv('input/nice_feature.csv',header=None,index_col=0)\n",
    "\n",
    "target = 'label'\n",
    "IDcol = 'id'\n",
    "ids = test['id'].values\n",
    "\n",
    "all_feature = [x for x in train.columns if x not in [target]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_imp_place20=pd.read_csv('input/feature_imp_place20.csv')\n",
    "\n",
    "predictors = [ x for x in all_feature if (x in nice_feature.index)|(x in feature_imp_place20.feature.values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0       6530\n",
       "1000.0     515\n",
       "1500.0     335\n",
       "2000.0     239\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Number of normal transactions: ', 1089)\n",
      "('Number of fraud transactions: ', 1089)\n",
      "('Total number of transactions in resampled data: ', 2178)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# tips：\n",
    "# 尝试欠采样和smoth采样\n",
    "\n",
    "# undersample\n",
    "train.to_csv('input/train_7r07part.csv',index=False)\n",
    "train=pd.read_csv('input/train_7r07part.csv')\n",
    "\n",
    "number_records_fraud = len(train[train.label != 0])\n",
    "fraud_indices = np.array(train[train.label != 0].index)\n",
    "\n",
    "# Picking the indices of the normal classes\n",
    "normal_indices = train[train.label == 0].index\n",
    "\n",
    "# Out of the indices we picked, randomly select \"x\" number (number_records_fraud)\n",
    "random_normal_indices = np.random.choice(normal_indices, number_records_fraud, replace = False)\n",
    "random_normal_indices = np.array(random_normal_indices)\n",
    "\n",
    "# Appending the 2 indices\n",
    "under_sample_indices = np.concatenate([fraud_indices,random_normal_indices])\n",
    "\n",
    "# Under sample dataset\n",
    "under_sample_data = train.iloc[under_sample_indices,:]\n",
    "\n",
    "X_undersample = under_sample_data.ix[:, under_sample_data.columns != 'label']\n",
    "y_undersample = under_sample_data.ix[:, under_sample_data.columns == 'label']\n",
    "\n",
    "# Showing ratio\n",
    "print(\"Number of normal transactions: \", len(under_sample_data[under_sample_data.label == 0]))\n",
    "print(\"Number of fraud transactions: \", len(under_sample_data[under_sample_data.label != 0]))\n",
    "print(\"Total number of transactions in resampled data: \", len(under_sample_data))\n",
    "\n",
    "\n",
    "# Oversample\n",
    "Oversampling1000 = train.loc[train.label == 1000]\n",
    "Oversampling1500 = train.loc[train.label == 1500]\n",
    "Oversampling2000 = train.loc[train.label == 2000]\n",
    "'''\n",
    "for i in range(6):\n",
    "    train = train.append(Oversampling1000)\n",
    "for j in range(10):\n",
    "    train = train.append(Oversampling1500)\n",
    "for k in range(12):\n",
    "    train = train.append(Oversampling2000)\n",
    "'''\n",
    "for i in range(5):\n",
    "    train = train.append(Oversampling1000)\n",
    "for j in range(8):\n",
    "    train = train.append(Oversampling1500)\n",
    "for k in range(10):\n",
    "    train = train.append(Oversampling2000)\n",
    "    \n",
    "train = train[train['label'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15264, 432)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "r_scale=preprocessing.RobustScaler()\n",
    "\n",
    "train_scaled=r_scale.fit_transform(train[predictors])\n",
    "test_scaled=r_scale.transform(test[predictors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.031658650391196586"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf=XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42)\n",
    "clf = clf.fit(train[predictors],train[target])\n",
    "result = clf.predict(test[predictors])\n",
    "\n",
    "f1_macro(y_test,result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def f1_macro0_drop(label_truth, predictions):\n",
    "    df=pd.DataFrame(columns=[\"subsidy_x\",\"subsidy_y\"])\n",
    "    df.subsidy_y=predictions\n",
    "    df.subsidy_x=np.array(label_truth)\n",
    "    df.subsidy_y = df.subsidy_y.apply(lambda x:int(x))\n",
    "\n",
    "    \n",
    "    correct = df[df['subsidy_x'] == df['subsidy_y']]\n",
    "    s = 0\n",
    "    for i in [1000, 1500, 2000]:\n",
    "        if sum(df['subsidy_x'] == i)!=0:\n",
    "            r = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_x'] == i)\n",
    "        else: \n",
    "            r=0\n",
    "        if sum(df['subsidy_y'] == i)!=0:\n",
    "            p = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_y'] == i)        \n",
    "        else:\n",
    "            p=0\n",
    "            \n",
    "        if (r+p)!=0:\n",
    "            f = r*p*2/(r+p)\n",
    "        else:\n",
    "            f=0\n",
    "        if not np.isnan(f):\n",
    "            s += (float(sum(df['subsidy_x'] == i))/df.shape[0])*f\n",
    "    \n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "my_score=make_scorer(score_func=f1_macro0_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 参数调节\n",
    "\n",
    "from sklearn import cross_validation, metrics  \n",
    "from sklearn.model_selection import GridSearchCV \n",
    "\n",
    "def modelfit(alg, dtrain, predictors, target,performCV=True, printFeatureImportance=False, cv_folds=3):\n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(dtrain[predictors], dtrain[target])\n",
    "\n",
    "    #Predict training set:\n",
    "    dtrain_predictions = alg.predict(dtrain[predictors])\n",
    "    dtrain_predprob = alg.predict_proba(dtrain[predictors])[:,1]\n",
    "\n",
    "    #Perform cross-validation:\n",
    "    if performCV:\n",
    "        cv_score = cross_validation.cross_val_score(alg, dtrain[predictors], dtrain[target], cv=cv_folds, scoring=my_score)\n",
    "\n",
    "    #Print model report:\n",
    "    print \"\\nModel Report\"\n",
    "    print \"Accuracy : %.4g\" % metrics.accuracy_score(dtrain[target].values, dtrain_predictions)\n",
    "    \n",
    "    if performCV:\n",
    "        print \"CV Score : Mean - %.7g | Std - %.7g | Min - %.7g | Max - %.7g\" % (np.mean(cv_score),np.std(cv_score),np.min(cv_score),np.max(cv_score))\n",
    "\n",
    "    #Print Feature Importance:\n",
    "    if printFeatureImportance:\n",
    "        feat_imp = pd.Series(alg.feature_importances_, predictors).sort_values(ascending=False)\n",
    "        feat_imp.plot(kind='bar', title='Feature Importances')\n",
    "        plt.ylabel('Feature Importance Score')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.8563\n",
      "CV Score : Mean - 0.1463062 | Std - 0.01041427 | Min - 0.1318068 | Max - 0.1557945\n"
     ]
    }
   ],
   "source": [
    "gbm0 = GradientBoostingClassifier(random_state=10)\n",
    "modelfit(gbm0, under_sample_data, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def Oversampling(train):\n",
    "    # Oversample\n",
    "    Oversampling1000 = train.loc[train.label == 1000]\n",
    "    Oversampling1500 = train.loc[train.label == 1500]\n",
    "    Oversampling2000 = train.loc[train.label == 2000]\n",
    "\n",
    "    for i in range(5):\n",
    "        train = train.append(Oversampling1000)\n",
    "    for j in range(8):\n",
    "        train = train.append(Oversampling1500)\n",
    "    for k in range(10):\n",
    "        train = train.append(Oversampling2000)\n",
    "    \n",
    "    train = train[train['label'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:662: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: 0.12288, std: 0.01573, params: {'n_estimators': 20},\n",
       "  mean: 0.12506, std: 0.00925, params: {'n_estimators': 30},\n",
       "  mean: 0.13755, std: 0.01015, params: {'n_estimators': 40},\n",
       "  mean: 0.13984, std: 0.01478, params: {'n_estimators': 50},\n",
       "  mean: 0.13868, std: 0.01559, params: {'n_estimators': 60},\n",
       "  mean: 0.14296, std: 0.01391, params: {'n_estimators': 70},\n",
       "  mean: 0.14423, std: 0.01627, params: {'n_estimators': 80}],\n",
       " {'n_estimators': 80},\n",
       " 0.14422876456755157)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {'n_estimators':range(20,81,10)}\n",
    "gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, min_samples_split=80,min_samples_leaf=8,max_depth=3,max_features='sqrt',subsample=0.8,random_state=10), \n",
    "param_grid = param_test1, scoring=my_score,n_jobs=4,iid=False, cv=5)\n",
    "#gsearch1.fit(train[predictors],train[target])\n",
    "\n",
    "gsearch1.fit(X_undersample[predictors],y_undersample[target])\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.8655\n",
      "CV Score : Mean - 0.001894665 | Std - 0.00378933 | Min - 0 | Max - 0.009473324\n"
     ]
    }
   ],
   "source": [
    "xgb0 = XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42,base_score=2000)\n",
    "modelfit(xgb0, train_test, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.985\n",
      "CV Score : Mean - 0.002591079 | Std - 0.004933227 | Min - 0 | Max - 0.01244976\n"
     ]
    }
   ],
   "source": [
    "rf0=RandomForestClassifier()\n",
    "modelfit(rf0, train_test, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.987\n",
      "CV Score : Mean - 0.003399415 | Std - 0.005718429 | Min - 0 | Max - 0.01478254\n"
     ]
    }
   ],
   "source": [
    "rf0=RandomForestClassifier(class_weight='balanced')\n",
    "modelfit(rf0, train_test, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## why the score of it is higher?\n",
    "## Because of resample.After resample,the dis of dataset had changed.\n",
    "# 0.02779836483035414\n",
    "\n",
    "xgb0 = XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42)\n",
    "modelfit(xgb0, under_sample_data, predictors,target)\n",
    "\n",
    "param_test1 = {'n_estimators':range(20,81,10)}\n",
    "gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, min_samples_split=80,min_samples_leaf=8,max_depth=3,max_features='sqrt',subsample=0.8,random_state=10), \n",
    "param_grid = param_test1, scoring=my_score,n_jobs=4,iid=False, cv=5)\n",
    "#gsearch1.fit(train[predictors],train[target])\n",
    "gsearch1.fit(X_undersample,y_undersample)\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Feature importance\n",
    "'''\n",
    "clf=XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42)\n",
    "clf = clf.fit(train[predictors],train[target])\n",
    "result = clf.predict_proba(test[predictors])\n",
    "f1_macro(y_test,result)\n",
    "\n",
    "feat_imp=pd.Series(clf.booster().get_fscore()).sort_values(ascending=False)\n",
    "feat_imp.plot(kind='bar', title='Feature Importances')\n",
    "feat_imp.tail(50)\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "## Evaluation\n",
    "def score(df):\n",
    "    # df有三列，ID:学生ID,subsidy_x:实际奖学金金额,subsidy_y:预测奖学金金额\n",
    "    correct = test_result[test_result['subsidy_x'] == test_result['subsidy_y']]\n",
    "    s = 0\n",
    "    for i in [1000, 1500, 2000]:\n",
    "        r = float(sum(correct['subsidy_y'] == i))/sum(test_result['subsidy_x'] == i)\n",
    "        p = float(sum(correct['subsidy_y'] == i))/sum(test_result['subsidy_y'] == i)\n",
    "        f = r*p*2/(r+p)\n",
    "        if not np.isnan(f):\n",
    "            s += (float(sum(test_result['subsidy_x'] == i))/test_result.shape[0])*f\n",
    "    print(s)\n",
    "\n",
    "test_result = pd.DataFrame(columns=[\"studentid\",\"subsidy_x\",\"subsidy_y\"])\n",
    "test_result.studentid = ids\n",
    "\n",
    "test_result.subsidy_x =np.array(y_test)\n",
    "test_result.subsidy_y = result\n",
    "test_result.subsidy_y = test_result.subsidy_y.apply(lambda x:int(x))\n",
    "\n",
    "score(test_result)\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
